package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractLoginDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.ZuiChengDuTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.*;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.chrome.ChromeDriver;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.net.URLDecoder;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.*;

/**
 * Created by 夏茂轩 on 2016/12/26.
 * @description 最成都下载器
 */
public class ZuiChengduDownLoader extends AbstractLoginDownLoader implements Cloneable{
    private Logger logger =Logger.getLogger(ZuiChengduDownLoader.class);
    private Map<String,File> files=null;

    public Map<String,File> getFiles(){

        return files;
    }
    /**
     * @author 夏茂轩
     * @date 2017/1/15 22:15
     * @description 最成都论坛的各个模块
     */
    private String[] modules={
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=44",//创业板块
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=11",//时尚生活
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=34",//数码时代
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=32",//房产装修
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=33",//汽车生活
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=39",//牙尖18怪
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=18",//同你在一起
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=22",//成都城建
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=15",//成都交通
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=19",//城市城际
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=42",//人文城市
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=41",//好摄之图
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=10",//认证商家
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=40",//跳蚤市场
            "http://www.chengtu.com/forum.php?mod=forumdisplay&fid=17"//论坛管理
     };

    @Override
    protected HashSet<Cookie> login() {

        ChromeDriver driver = null;
        try {
            driver= ChromeDriverUtils.getDriver();
            driver.get("http://www.chengtu.com/");
            try {
                Thread.sleep(6000);
            } catch (InterruptedException e) {
                e.printStackTrace();
                logger.error(e.getMessage(),e);
            }

            return (HashSet<Cookie>) driver.manage().getCookies();
        } finally {
            driver.quit();
        }
    }

    /**
     * @author 夏茂轩
     * @date 2017/1/11 14:26
     * @description 生成最成都爬虫任务并添加到任务队列
     */
    public void generateTask(){
        for(int i=0;i<modules.length;i++){
            try {
                Document document = get(modules[i]);
                String page=document.select("#fd_page_top > div > label > span").attr("title");
                int acPage=Integer.parseInt(page.substring(2,page.length()-2));
                Elements elements=document.select("tbody[id^='normalthread_'] a.xst");
                for(int j=0;j<elements.size();j++){
                    String forumUrl=elements.get(j).attr("href");
                    ZuiChengDuTask task =new ZuiChengDuTask(URLDecoder.decode(forumUrl,"UTF-8"));
                    CommonTaskManager.add("ZuiChengDu",task);
                }
                /*for(int k=2;k<acPage+1;k++){
                    String url=modules[i]+"&page="+k;
                    Document doc=get(url);
                    Elements elements2=doc.select("tbody[id^='normalthread_'] a.xst");
                    for(int j=0;j<elements2.size();j++){
                        String forumUrl=elements2.get(j).attr("href");
                        ZuiChengDuTask task =new ZuiChengDuTask(URLDecoder.decode(forumUrl,"UTF-8"));
                        CommonTaskManager.add("ZuiChengDu",task);
                    }
                }*/
            } catch (Exception e) {
                e.printStackTrace();
                logger.error(e.getMessage(),e);
                continue;
            }
        }
    }

    /**
     * @author 夏茂轩
     * @date 2017/1/11 14:25
     * @description 根据帖子的URL爬取言论及其回复
     * @param url 帖子链接
     * @return JSONObject
     */
    public JSONObject crawlItem(String url){
        files=new ConcurrentHashMap<>();
        JSONObject item=new JSONObject();
        JSONObject data=new JSONObject();
        int page=1;
        try {
            Document document = null;
            boolean flag = true;
            long startTime = System.currentTimeMillis();
            long nowTime = startTime;
            while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("ZuiChengDu.TimeOut"))) {

                try {
                    int threadNum=Integer.valueOf(PropertiesUtil.getVal("ZuiChengDu.ReplyThreadCount"));
//                Document document=get(url);
                    Page pageContent = null;
                    pageContent = getHtml(url);
                    if (null == pageContent) {

                    } else if (pageContent.getStatusCode() == 0) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                    } else if (pageContent.getStatusCode() == 200) {
                        try {
                            flag = false;
                            document = pageContent.getHtml().getDocument();
                            String title = document.title();
                            Element pageInfo = document.select(".posts_info").get(0);
                            int read_count = Integer.parseInt(pageInfo.select(".rf").get(0).select("strong").get(0).text());//阅读量
                            int reply_count = Integer.parseInt(pageInfo.select(".rf").get(0).select("strong").get(1).text());//回复量
                            data.put("reply_count", reply_count);
                            data.put("read_count", read_count);
                            data.put("title", (title));
                            data.put("url", url);
                            page = (int) Math.ceil(((double) reply_count + 1) / 10d);//页数
                            String publish_time = "";//发布时间
                            if (pageInfo.select(".lf").select("span").hasAttr("title")) {
                                publish_time = pageInfo.select(".lf").select("span").attr("title") + ":00";
                            } else {
                                publish_time = getTime(pageInfo.select(".lf").select("span").text()) + ":00";
                            }
                            data.put("publish_time", publish_time);
                            Date now = new Date();
                            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                            data.put("spider_time", sdf.format(now));
                            Element forumEntity = document.select(".rtv").get(0);
                            String content = "";
                            JSONArray img_urls = new JSONArray();
                            if (forumEntity.select(".t_fsz").size() > 0) {
                                content = forumEntity.select(".t_fsz").text();
                                Elements imgElements=forumEntity.select(".t_fsz").select("img[file]");
                                for(int imageIndex=0;imageIndex<imgElements.size();imageIndex++){
                                    Element eImg=imgElements.get(imageIndex);
                                    String img_url=eImg.absUrl("file");
                                    img_urls.add(img_url);
                                    System.out.printf(img_url);
                              }
                            } else {
                                content = forumEntity.select(".pct").text();
                                Elements imgElements=forumEntity.select(".pct").select("img[file]");
                                for(int imageIndex=0;imageIndex<imgElements.size();imageIndex++){
                                    Element eImg=imgElements.get(imageIndex);
                                    String img_url=eImg.absUrl("file");
                                    img_urls.add(img_url);
                                    System.out.printf(img_url);
                                }
                            }
                            try {
                                String reproduce = forumEntity.select(".mbm").select("a").get(4).select("i").select("span").text();
                                String like = forumEntity.select(".mbm").select("a").get(6).select("i").select("span").text();
                                String dislike = forumEntity.select(".mbm").select("a").get(7).select("i").select("span").text();
                                if (!"".equals(reproduce) && reproduce != null) {
                                    int reproduce_count = Integer.parseInt(reproduce);
                                    data.put("reproduce_count", reproduce_count);
                                }
                                if (!"".equals(like) && like != null) {
                                    int like_count = Integer.parseInt(like);
                                    data.put("like_count", like_count);
                                }
                                if (!"".equals(dislike) && dislike != null) {
                                    int dislike_count = Integer.parseInt(dislike);
                                    data.put("dislike_count", dislike_count);
                                }
                            } catch (Exception e) {
                                logger.error(e.getMessage(),e);
                            }
                            Element userInfo = forumEntity.select(".pls").get(0);
                            String publish_user_id = getUrlParams(userInfo.select(".authi").select("a").get(0).attr("href")).get("uid").toString();//发布者的ID
                            String publish_user = userInfo.select(".authi").select("a").get(0).text();//发布用户
                            String publish_user_photo = String.format("http://accounts.chengtu.com/avatar.php?uid=%s", publish_user_id);
                            String id = getUrlParams(url).get("tid").toString();//帖子的Id
                            data.put("content", (content));
                            data.put("publish_user_id", publish_user_id);
                            data.put("publish_user", (publish_user));
                            data.put("publish_user_photo", publish_user_photo);
                            data.put("id", id);
                            data.put("img_urls",img_urls);
                            long tsp = DateUtils.dateToTimestamp(DateUtils.parseDate(publish_time)).getTime();
                            String filePrefix = "ZuiChengdu";//文件前缀
                            String fileName = filePrefix + "_" + tsp + "_" + publish_user_id + "_" + id;
                            String publishDate = DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-", "-");
                            String speechesId = "ZuiChengdu_speeches_" + publishDate + "_" + fileName;
                            File file = FileUtils.createTempFile(speechesId + "_pageNo_1", document.html().toString());
                            files.put(speechesId + "_pageNo_1", file);
                            //帖子首页的文件
                            //将本页面的回复添加上去
                            Elements replyEntities = document.select(".rtv:gt(1)");
                            JSONArray reply_nodes = new JSONArray();
                            for (int i = 0; i < replyEntities.size(); i++) {
                                Element replyUserInfo = replyEntities.get(i).select(".pls").get(0);
                                JSONObject reply = new JSONObject();
                                String replyUser = replyUserInfo.select(".authi").select("a").get(0).text();
                                String homeUrl = replyUserInfo.select(".authi").select("a").get(0).attr("href");
                                String replyUserId = getUrlParams(homeUrl).get("uid").toString();
                                String replyUserPhoto = String.format("http://accounts.chengtu.com/avatar.php?uid=%s", replyUserId);
                                String replyContent = "";
                                JSONArray reply_img_urls=new JSONArray();
                                if (replyEntities.get(i).select(".t_fsz").size() > 0) {
                                    replyContent = replyEntities.get(i).select(".t_fsz").text();
                                    Elements imgElements=replyEntities.select(".t_fsz").select("img[file]");
                                    for(int imageIndex=0;imageIndex<imgElements.size();imageIndex++){
                                        Element eImg=imgElements.get(imageIndex);
                                        String img_url=eImg.absUrl("file");
                                        reply_img_urls.add(img_url);
                                    }

                                } else {
                                    replyContent = replyEntities.get(i).select(".pct").text();
                                    Elements imgElements=replyEntities.select(".pct").select("img[file]");
                                    for(int imageIndex=0;imageIndex<imgElements.size();imageIndex++){
                                        Element eImg=imgElements.get(imageIndex);
                                        String img_url=eImg.absUrl("file");
                                        reply_img_urls.add(img_url);
                                    }
                                }
                                String replyTime = getTime(replyEntities.get(i).select(".authi").html());
                                String replyUrl = url + "#" + replyEntities.get(i).id();
                                reply.put("publish_user", replyUser);
                                reply.put("publish_user_id", replyUserId);
                                reply.put("publish_user_photo", replyUserPhoto);
                                reply.put("content", replyContent);
                                reply.put("publish_time", replyTime);
                                reply.put("url", replyUrl);
                                reply.put("parent_id", id);
                                reply.put("ancestor_id", id);
                                reply.put("img_urls",reply_img_urls);
                                reply_nodes.add(reply);
                            }
                            JSONArray replys = new JSONArray();
                            List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
                            if (page >= 2) {
                                List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
                                ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
                                try {
                                    ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, page, threadNum);
                                    for (int i = 0; i < splitPages.size(); i++) {
                                        ZuiChengduDownLoader downLoader = this.cloneDownLoader();
                                        ReplyCrawler callable = new ReplyCrawler(downLoader, splitPages.get(i), url, id, speechesId);
                                        FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
                                        futureTasks.add(futureTask);
                                        executorService.submit(futureTask);
                                    }
                                    int count = 0;
                                    for (FutureTask<JSONArray> futureTask : futureTasks) {
                                        JSONArray replyArrays = futureTask.get();
                                        for (int i = 0; i < replyArrays.size(); i++) {
                                            JSONObject object = replyArrays.getJSONObject(i);
                                            replys.add(object);
                                        }
                                    }
                                    if (replys.size() > 0 && replys != null) {
                                        for (int i = 0; i < replys.size(); i++) {
                                            JSONObject reply_object = replys.getJSONObject(i);
                                            reply_nodes.add(reply_object);
                                        }
                                    }
                                    if (reply_nodes.size() > 0 && reply_nodes != null) {
                                        for (int i = 0; i < reply_nodes.size(); i++) {
                                            JSONObject reply_object = reply_nodes.getJSONObject(i);
                                            Map<String, Object> map = reply_object;
                                            list.add(map);
                                        }
                                    }
                                    Collections.sort(list, new Comparator<Map<String, Object>>() {

                                        public int compare(Map<String, Object> o1, Map<String, Object> o2) {
                                            String s1 = o1.get("publish_time").toString();
                                            String s2 = o2.get("publish_time").toString();
                                            int flag = s1.compareTo(s2);
                                            return flag;
                                        }
                                    });
                                } finally {
                                    executorService.shutdown();
                                }
                            } else {
                                if (reply_nodes.size() > 0 && reply_nodes != null) {
                                    for (int i = 0; i < reply_nodes.size(); i++) {
                                        JSONObject reply_object = reply_nodes.getJSONObject(i);
                                        Map<String, Object> map = reply_object;
                                        list.add(map);
                                    }
                                }
                            }
                            data.put("reply_nodes", list);
                        } catch (Exception e) {
                            logger.error(e.getMessage(),e);
                        }finally {
                            flag=false;
                        }
                    }
                } catch (Exception e) {
                    logger.error(e.getMessage(),e);
                }finally {
                    if(flag==true){
                        item = new JSONObject();
                        data = new JSONObject();
                    }
                    nowTime = System.currentTimeMillis();
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
            logger.error(e.getMessage(),e);
        }
        item.put("data",data);
        return  item;
    }

    public ZuiChengduDownLoader cloneDownLoader() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        ZuiChengduDownLoader loader =new ZuiChengduDownLoader();
        loader.cookies= (HashSet<Cookie>) this.cookies.clone();
        loader.headers= (HashMap<String, String>) this.headers.clone();
        return loader;
    }

    /**
     * @author 夏茂轩
     * @date 2017/1/15 22:14
     * @description 回复爬取，内部类
     */
    public class ReplyCrawler implements Callable<JSONArray>{
        private ZuiChengduDownLoader downLoader;
        private ArrayList<Integer> pages;
        private String url;
        private String parent_id;
        private String speeches_id;
        @Override
        public JSONArray call() throws Exception {
            JSONArray array =new JSONArray();
            for(int k=0;k<pages.size();k++){
                JSONArray reply_nodes =new JSONArray();
                try {
                    boolean flag = true;
                    long startTime = System.currentTimeMillis();
                    long nowTime = startTime;
                    while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("ZuiChengDu.TimeOut"))) {
                        try {
                            String getUrl=url+"&page="+pages.get(k);
                            Document document = null;
                            Page pageContent = null;//getHtml(getUrl, ProxyPool.proxy());
                            pageContent = downLoader.getHtml(getUrl);
                            if (null == pageContent) {

                            } else if (pageContent.getStatusCode() == 0) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                            } else if (pageContent.getStatusCode() == 200) {
                                flag = false;
                                document = pageContent.getHtml().getDocument();
                                File file=FileUtils.createTempFile(speeches_id+"_pageNo_"+pages.get(k),document.html().toString());
                                files.put(speeches_id+"_pageNo_"+pages.get(k),file);
                                Elements replyEntities=document.select(".rtv");

                                for(int i=0;i<replyEntities.size();i++) {
                                    try {
                                        Element replyUserInfo = replyEntities.get(i).select(".pls").get(0);
                                        JSONObject reply = new JSONObject();
                                        String replyUser = replyUserInfo.select(".authi").select("a").get(0).text();
                                        String homeUrl = replyUserInfo.select(".authi").select("a").get(0).attr("href");
                                        String replyUserId = getUrlParams(homeUrl).get("uid").toString();
                                        String replyUserPhoto = String.format("http://accounts.chengtu.com/avatar.php?uid=%s", replyUserId);
                                        String replyContent = "";
                                        JSONArray reply_img_urls=new JSONArray();
                                        if (replyEntities.get(i).select(".t_fsz").size() > 0) {
                                            replyContent = replyEntities.get(i).select(".t_fsz").text();
                                            Elements imgElements=replyEntities.select(".t_fsz").select("img[file]");
                                            for(int imageIndex=0;imageIndex<imgElements.size();imageIndex++){
                                                Element eImg=imgElements.get(imageIndex);
                                                String img_url=eImg.absUrl("file");
                                                reply_img_urls.add(img_url);
                                            }
                                        } else {
                                            replyContent = replyEntities.get(i).select(".pct").text();
                                            Elements imgElements=replyEntities.select(".pct").select("img[file]");
                                            for(int imageIndex=0;imageIndex<imgElements.size();imageIndex++){
                                                Element eImg=imgElements.get(imageIndex);
                                                String img_url=eImg.absUrl("file");
                                                reply_img_urls.add(img_url);
                                            }
                                        }
                                        String replyTime = getTime(replyEntities.get(i).select(".authi").html());
                                        String replyUrl = getUrl + "#" + replyEntities.get(i).id();
                                        reply.put("publish_user", (replyUser));
                                        reply.put("publish_user_id", replyUserId);
                                        reply.put("publish_user_photo", replyUserPhoto);
                                        reply.put("content", (replyContent));
                                        reply.put("publish_time", replyTime);
                                        reply.put("url", replyUrl);
                                        reply.put("parent_id", parent_id);
                                        reply.put("ancestor_id", parent_id);
                                        reply.put("img_urls",reply_img_urls);
                                        reply_nodes.add(reply);
                                    } catch (Exception e) {
                                        logger.error(e.getMessage(), e);
                                        continue;//因为禁言无法获取的情况
                                    }
                                }
                            }
                        } catch (Exception e) {
                            logger.error(e.getMessage(), e);
                        }finally {
                            if(flag==false){
                                array.addAll(reply_nodes);
                            }
                            nowTime = System.currentTimeMillis();
                        }
                    }
                } catch (Exception e) {
                    logger.error(e);
                    e.printStackTrace();
                    continue;
                }
            }
            return array;
        }
        public ReplyCrawler(ZuiChengduDownLoader downLoader, ArrayList<Integer> pages,String url,String parent_id,String speeches_id) {
            this.downLoader = downLoader;
            this.pages = pages;
            this.url=url;
            this.parent_id = parent_id;
            this.speeches_id=speeches_id;
        }
    }
}
